#include "sequences.h"
#include <strstream>

const double kCRPTheta = 1000;
const double kCRPAlpha = 0;

bool LoadSequence(istream &in, Sequence & s) {
  s.v_.clear();
  char c;
  if (!(in >> c)) return false;
  CHECK(c == 'A');
  in >> s.id_;
  bool ok = true;
  in >> c;
  CHECK(c == ',');
  while (in.peek() != '\n') {
    string tok;
    while (in.peek() != ',') tok += in.get();
    istringstream istr(tok);
    int64 i;
    istr >> i;
    ostringstream ostr;
    ostr << i;
    if (ostr.str() != tok) ok = false;
    if (abs(i) >= (1LL << 60)) ok = false;
    if (ok) s.v_.push_back(i);
    CHECK(in.get() == ',');
  }
  in >> ws;
  return true;
}

bool operator <(const Sequence &a, const Sequence &b) {
  uint32 shared_length = min(TrainingPortion(a.v_.size()), 
			     TrainingPortion(b.v_.size()));
  for (uint32 i=0; i<shared_length; i++) {
    if (a.v_[i] < b.v_[i]) return true;
    if (b.v_[i] < a.v_[i]) return false;
  }
  if (TrainingPortion(a.v_.size()) < TrainingPortion(b.v_.size())) 
    return true;
  return false;
}
bool IsPrefix(const Sequence &a, const Sequence &b) {
  if (TrainingPortion(b.v_.size()) < TrainingPortion(a.v_.size())) return false;
  for (uint32 i=0; i < TrainingPortion(a.v_.size()); i++) {
    if (a.v_[i] != b.v_[i]) return false;
  }
  return true;
}
bool operator ==(const Sequence &a, const Sequence &b) {
  if (TrainingPortion(b.v_.size()) != TrainingPortion(a.v_.size())) 
    return false;
  for (uint32 i=0; i < TrainingPortion(a.v_.size()); i++) {  
    if (a.v_[i] != b.v_[i]) return false;
  }
  return true;
}

void SequenceIndex::Load() {
  // CHECK(false);
  cout << "Loading sequences " << endl;
  ifstream input("sequences.bin", ios::in | ios::binary);
  input.read((char *)&num_sequences_, sizeof(int32));
  original_ids_ = new int32[num_sequences_];
  lengths_ = new int32[num_sequences_];
  sequences_ = new int64*[num_sequences_];
  descriptions_ = new char *[num_sequences_ + 1];
  for (int i=0; i<num_sequences_; i++) {
    input.read((char *)(original_ids_ + i), sizeof(int32));
    input.read((char *)(lengths_ + i), sizeof(int32));
    sequences_[i] = new int64[lengths_[i]];
    input.read((char *)(sequences_[i]), lengths_[i] * sizeof(int64));
    int32 desc_size;
    input.read((char *)&desc_size, sizeof(int32));
    descriptions_[i] = new char[desc_size];
    input.read(descriptions_[i], desc_size);
  }
}

void SequenceSearch::FlagReferences(int32 shard, Frontier *frontier) {
  for (uint64 i=0; i<NumSequences(); i++)
    frontier->AddObject(seq_info_[i].code_);
}
void SequenceSearch::ObjectsDeleted(int32 shard) {
  for (uint64 i=0; i<NumSequences(); i++) {
    CHECK(!WasDeleted(seq_info_[i].code_));
    seq_info_[i].code_ = NewLocation(seq_info_[i].code_);
  }
}

SequenceSearch::SequenceSearch() : gdl_(kCRPTheta, kCRPAlpha) {
  dl_ = 0;
  cache_size_ = 12345679;
  code_seq_cache_ = new pair<uint64, pair<int32, int32> >[cache_size_];
  index_.Load();
  G_ALLOC.AddDependant(this);
  pthread_mutex_init(&lock_, NULL);
  num_correct_ = num_predicted_ = 0;
  seq_info_ = new SeqInfo[index_.num_sequences_];
  for (int32 i=0; i < index_.num_sequences_; i++) {
    uint64 dl = 0;
    int64 * seq = index_.sequences_[i];
    int32 length = index_.lengths_[i];
    for (int32 j=0; j < (int32)TrainingPortion(length); j++)
      dl += IntegerDL(seq[j]);
    seq_info_[i].original_dl_ = dl;
    dl_ += dl;
  }
}

// Loads the object_seq_cache_ and tells you if you'd have positive utility
pair<int32, int32>  SequenceSearch::CodeToSequence(Object *& code,
						   Vinterpreter *interp) {
  uint64 hash = Hash(code);
  pair<uint64, pair<int32, int32> > & look = 
    code_seq_cache_[hash % cache_size_];
  if (look.first == hash) return look.second;
  
  Interpret in;
  in.interp_ = interp;
  in.code_ = &code;
  pair<int32, int32> ret = index_.Search(in);
  
  // Update the hash if we had enough time to run
  if (!interp->Overtime()) {
    look = make_pair(hash, ret); 
  }
  return ret;
}

Object * SequenceSearch::Data(Object *&param, ThreadAllocator *a) {
  if (GetType(param) != INTEGER) return NULL;
  uint64 id = GetInteger(param);
  if (id >= NumSequences()) return NULL;
  vector<Object *> ret;
  int64 * seq = index_.GetSequence(id);
  int64 length  = index_.GetSequenceLength(id);
  for (int64 i=0; i<length; i++) {
    ret.push_back(MakeInteger(seq[i]));
  }
  return a->MakeArray(ret);  
}

pthread_t g_lock_holder;


// If update is false, return 0.0 if it matches no sequence, 1.0 if it does
// Otherwise, we're really updating and return the real diff in dl

double SequenceSearch::Test(Object *& code, 
			    Vinterpreter *interp, 
			    bool update) {

  // Don't test if it's in the cache and update is false
  // This seems like a pretty big hack but it pays off
  if (!update) {
    uint64 hash = Hash(code);
    pair<uint64, pair<int32, int32> > look = 
      code_seq_cache_[hash % cache_size_];
    if (look.first == hash) return 0.0;
  }

  // Initially test to see which sequence it matches
  // and return quickly if update is false
  pair<int32, int32> p = CodeToSequence(code, interp);
  int32 seq = p.first;
  if (seq == -1) return 0.0;
  if (!update) return 1.0;
  int32 correct = p.second;
  
  // we are updating.
  char code_string[20000];
  ostrstream ostr(code_string, 10000);
  ostr << code;
  ostr << (unsigned char) 0;
  // string code_string = ToString(code);
  
  MutexLocker locker(&lock_);
  int64 old_dl = dl_;
  SeqInfo old_seq_info = seq_info_[seq];
  SetSeqInfo(seq, code, correct);
  if (dl_ >= old_dl) {
    SetSeqInfo(seq, old_seq_info.code_, old_seq_info.correct_);
    CHECK(dl_ == old_dl);
    return 0.0;
  }
  CheckEverMatched(seq);
  double d_utility = (old_dl - dl_) / 1e6;
  store_ << seq << " " << correct << " " << code_string << endl;
  if (!old_seq_info.code_) cout << "New Match" << endl;
  cout << "correct / predicted (new): " 
       << num_correct_ << " / " << num_predicted_ 
       << " ( " << new_matches_this_run_ << " )" << endl;
  cout << "code = " << code_string << endl;
  if (!correct) cout << " INCORRECT" << endl;
  cout << "description = " << index_.GetDescription(seq) << endl;
  cout << "utility " << d_utility << endl;
  cout << endl;
  
  return d_utility;
}

void SequenceSearch::LoadState(ThreadAllocator *ta) {
  ifstream input(StoreFileName().c_str());
  int32 num;
  int32 correct;
  Object * code;
  ifstream input2("ever_matched");
  ever_matched_ = vector<bool>(NumSequences());
  while (input2 >> num) {
    // input2.getline(buf, 1000000);
    ever_matched_[num] = true;
  }
  new_matches_this_run_ = 0;

  while (input >> num >> correct) {
    code = ReadObject(ta, input);
    SetSeqInfo(num, code, correct);
    CheckEverMatched(num);
  }
  store_.open(StoreFileName().c_str(), ios::app);
}

struct X {
  int64 operator ()(int32 x) { return x_;}
  X(int32 x) { x_ = x;}
  int32 x_;
};

void MakeBinarySequenceFile() {
  ifstream input("stripped");
  ofstream debug("debug-sorted");
  Sequence s;
  vector<Sequence> w;
  for (int i=0; i<200000; i++) {
    if (i%10000 == 0) cout << "loaded " << i << " Sequences " << endl;
    if (!LoadSequence(input, s)) break;
    if (s.v_.size() < 3) continue;
    w.push_back(s);
  }
  sort(w.begin(), w.end());
  set<int> bad;
  for (uint32 i=0; i+1<w.size(); i++) {
    if (w[i] == w[i+1]) bad.insert(i);
    else if (IsPrefix(w[i], w[i+1])) bad.insert(i);
    Sequence & seq = w[i];
    if (bad % (int)i)
      debug << "bad  ";
    else debug << "     ";
    debug << "ID:" << seq.id_ << "  ";
    for (uint32 j=0; j<seq.v_.size(); j++) 
      debug << seq.v_[j] << " ";
    debug << endl;
  }
  vector<Sequence> v;
  for (uint32 i=0; i<w.size(); i++) {
    if (!(bad % (int)i)) v.push_back(w[i]);
  }

  map<int, string> descriptions;
  ifstream desc("names");
  char buf[1001];
  char c;
  while (desc >> c) {
    CHECK(c == 'A');
    int id;
    desc >> id;
    desc.get();
    desc.getline(buf, 1001);
    descriptions[id] = "A" + itoa(id) + " " + buf;
  }

  ofstream out("sequences.bin", ios::out | ios::binary);
  int32 size = v.size();
  out.write((char *)&size, sizeof(int32));
  for (int i=0; i<size; i++) {
    int32 seq_size = v[i].v_.size();
    out.write((char *)&(v[i].id_), sizeof(int32));
    out.write((char *)&seq_size, sizeof(int32));
    out.write((char *)&(v[i].v_[0]), v[i].v_.size() * sizeof(int64));
    string description = descriptions[v[i].id_];
    int32 desc_size = description.size() + 1;
    out.write((char *)&desc_size, sizeof(int32));
    out.write(description.c_str(), desc_size);
  }
}



int sequences_main() {
  
  SequenceIndex t;
  t.Load();
  for (int i=0; i<10; i++) {
    // cout << " Testing function x=" << i << endl;
    vector<int32> results;
    X x(i);
    pair<int32, int32>  result = t.Search(x);
    if (result.first < 0) cout << "not found";
    else 
      cout << result.first << " " << result.second << " "  << t.GetDescription(result.first) << endl;
  }
  return 0;
 
}

